{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting datasets\r\n",
      "  Downloading datasets-2.3.2-py3-none-any.whl (362 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 362 kB 818 kB/s eta 0:00:01\r\n",
      "\u001B[?25hRequirement already satisfied: tqdm>=4.62.1 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from datasets) (4.64.0)\r\n",
      "Collecting multiprocess\r\n",
      "  Downloading multiprocess-0.70.13-py38-none-any.whl (131 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 131 kB 11.1 MB/s eta 0:00:01\r\n",
      "\u001B[?25hRequirement already satisfied: pandas in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from datasets) (1.3.5)\r\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from datasets) (0.6.0)\r\n",
      "Collecting dill<0.3.6\r\n",
      "  Downloading dill-0.3.5.1-py2.py3-none-any.whl (95 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 95 kB 7.8 MB/s  eta 0:00:01\r\n",
      "\u001B[?25hCollecting pyarrow>=6.0.0\r\n",
      "  Downloading pyarrow-8.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.4 MB)\r\n",
      "\u001B[K     |████████████████████████████████| 29.4 MB 1.1 MB/s  eta 0:00:01\r\n",
      "\u001B[?25hRequirement already satisfied: numpy>=1.17 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from datasets) (1.22.3)\r\n",
      "Collecting fsspec[http]>=2021.05.0\r\n",
      "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 140 kB 10.9 MB/s eta 0:00:01\r\n",
      "\u001B[?25hRequirement already satisfied: requests>=2.19.0 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from datasets) (2.27.1)\r\n",
      "Collecting xxhash\r\n",
      "  Downloading xxhash-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 212 kB 10.8 MB/s eta 0:00:01\r\n",
      "\u001B[?25hCollecting responses<0.19\r\n",
      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\r\n",
      "Requirement already satisfied: packaging in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from datasets) (21.3)\r\n",
      "Collecting aiohttp\r\n",
      "  Downloading aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.3 MB)\r\n",
      "\u001B[K     |████████████████████████████████| 1.3 MB 10.7 MB/s eta 0:00:01\r\n",
      "\u001B[?25hRequirement already satisfied: filelock in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.0)\r\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.2.0)\r\n",
      "Requirement already satisfied: pyyaml in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from packaging->datasets) (3.0.9)\r\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2.0.12)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.3)\r\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.9)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2022.5.18.1)\r\n",
      "Collecting multidict<7.0,>=4.5\r\n",
      "  Downloading multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 121 kB 8.7 MB/s eta 0:00:01\r\n",
      "\u001B[?25hRequirement already satisfied: attrs>=17.3.0 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from aiohttp->datasets) (21.4.0)\r\n",
      "Collecting yarl<2.0,>=1.0\r\n",
      "  Downloading yarl-1.7.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (308 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 308 kB 9.3 MB/s eta 0:00:01\r\n",
      "\u001B[?25hCollecting frozenlist>=1.1.1\r\n",
      "  Downloading frozenlist-1.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\r\n",
      "\u001B[K     |████████████████████████████████| 158 kB 8.7 MB/s eta 0:00:01\r\n",
      "\u001B[?25hCollecting aiosignal>=1.1.2\r\n",
      "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\r\n",
      "Collecting async-timeout<5.0,>=4.0.0a3\r\n",
      "  Using cached async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\r\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from pandas->datasets) (2022.1)\r\n",
      "Requirement already satisfied: six>=1.5 in /home/fanfanfeng/anaconda3/envs/nlp_tools/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.16.0)\r\n",
      "Installing collected packages: multidict, frozenlist, yarl, async-timeout, aiosignal, fsspec, dill, aiohttp, xxhash, responses, pyarrow, multiprocess, datasets\r\n",
      "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 datasets-2.3.2 dill-0.3.5.1 frozenlist-1.3.0 fsspec-2022.5.0 multidict-6.0.2 multiprocess-0.70.13 pyarrow-8.0.0 responses-0.18.0 xxhash-3.0.0 yarl-1.7.2\r\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "!set HTTP_PROXY=http://127.0.0.1:32849\n",
    "!set HTTPS_PROXY=http://127.0.0.1:32849"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset 1/plain_text to /home/fanfanfeng/.cache/huggingface/datasets/1/plain_text/1.0.0/a0b4f9ec1f3c675b156231b64c4cd9196bce58e3414b09eb235377193dd48f0d...\n"
     ]
    },
    {
     "data": {
      "text/plain": "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "5efc644aa1984e3480c35b064ac2dab0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Downloading data:   0%|          | 0.00/8.12M [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "8859673b34ac4812a1d4ef229d9ec68f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Downloading data:   0%|          | 0.00/1.05M [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "9f4c484abc7d42668ebaa443f5b1c9e9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "1619c9765c2642ca9a41e188ac770f13"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Generating train split: 0 examples [00:00, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a66a0b9e1c8d4ea0bb2ebb0ec87e65e8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Generating validation split: 0 examples [00:00, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "c898f8c6a9c54e4282360887ae4503b0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset 1 downloaded and prepared to /home/fanfanfeng/.cache/huggingface/datasets/1/plain_text/1.0.0/a0b4f9ec1f3c675b156231b64c4cd9196bce58e3414b09eb235377193dd48f0d. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/2 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "bc3841f478dd4d878f5c80c5497eafd9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "DatasetDict({\n    train: Dataset({\n        features: ['id', 'title', 'context', 'question', 'answers'],\n        num_rows: 87599\n    })\n    validation: Dataset({\n        features: ['id', 'title', 'context', 'question', 'answers'],\n        num_rows: 10570\n    })\n})"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "squad_v2 = False\n",
    "\n",
    "datasets = load_dataset(\"1.py\")\n",
    "datasets"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "{'id': '5733be284776f41900661182',\n 'title': 'University_of_Notre_Dame',\n 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.',\n 'question': 'To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?',\n 'answers': {'text': ['Saint Bernadette Soubirous'], 'answer_start': [515]}}"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets[\"train\"][0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "from datasets import ClassLabel,Sequence\n",
    "import random\n",
    "import pandas as pd\n",
    "from IPython.display import display,HTML\n",
    "\n",
    "def show_random_elements(dataset,num_examples=10):\n",
    "    assert num_examples <= len(dataset), \"can't pick more elements than there are in the dataset\"\n",
    "    picks = []\n",
    "    for _ in range(num_examples):\n",
    "        pick = random.randint(0,len(dataset) - 1)\n",
    "        while pick in picks:\n",
    "            pick = random.randint(0,len(dataset) - 1)\n",
    "        picks.append(pick)\n",
    "\n",
    "    df = pd.DataFrame(dataset[picks])\n",
    "    for column,tup in dataset.features.items():\n",
    "        if isinstance(tup,ClassLabel):\n",
    "            df[column] = df[column].transform(lambda i:tup.names[i])\n",
    "        elif isinstance(tup,Sequence) and isinstance(tup.feature,ClassLabel):\n",
    "            df[column] = df[column].transform(lambda x: [tup.feature.names[i] for i in x])\n",
    "\n",
    "    display(HTML(df.to_html()))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>title</th>\n      <th>context</th>\n      <th>question</th>\n      <th>answers</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>571a05b84faf5e1900b8a86d</td>\n      <td>Seattle</td>\n      <td>Prior to moving its headquarters to Chicago, aerospace manufacturer Boeing (#30) was the largest company based in Seattle. Its largest division is still headquartered in nearby Renton, and the company has large aircraft manufacturing plants in Everett and Renton, so it remains the largest private employer in the Seattle metropolitan area. Former Seattle Mayor Greg Nickels announced a desire to spark a new economic boom driven by the biotechnology industry in 2006. Major redevelopment of the South Lake Union neighborhood is underway, in an effort to attract new and established biotech companies to the city, joining biotech companies Corixa (acquired by GlaxoSmithKline), Immunex (now part of Amgen), Trubion, and ZymoGenetics. Vulcan Inc., the holding company of billionaire Paul Allen, is behind most of the development projects in the region. While some see the new development as an economic boon, others have criticized Nickels and the Seattle City Council for pandering to Allen's interests at taxpayers' expense. Also in 2006, Expansion Magazine ranked Seattle among the top 10 metropolitan areas in the nation for climates favorable to business expansion. In 2005, Forbes ranked Seattle as the most expensive American city for buying a house based on the local income levels. In 2013, however, the magazine ranked Seattle No. 9 on its list of the Best Places for Business and Careers.</td>\n      <td>Where in the Seattle area does Boeing have manufacturing plants?</td>\n      <td>{'text': ['Everett and Renton'], 'answer_start': [244]}</td>\n    </tr>\n  </tbody>\n</table>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_random_elements(datasets[\"train\"], num_examples=1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "44fdccaf20c8496caa931bc367c0aa96"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Downloading:   0%|          | 0.00/483 [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "65c63e9078c5442b944e358a1689c992"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "c68b2b6e521148a3814fc664d35e56be"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d61ffdd2e92041ed8356787fae57bac3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import  AutoTokenizer\n",
    "model_checkpoint = 'distilbert-base-uncased'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint,use_fast=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "[101,\n 2000,\n 3183,\n 2106,\n 1996,\n 6261,\n 2984,\n 9382,\n 3711,\n 1999,\n 8517,\n 1999,\n 10223,\n 26371,\n 2605,\n 1029,\n 102,\n 6549,\n 2135,\n 1010,\n 1996,\n 2082,\n 2038,\n 1037,\n 3234,\n 2839,\n 1012,\n 10234,\n 1996,\n 2364,\n 2311,\n 1005,\n 1055,\n 2751,\n 8514,\n 2003,\n 1037,\n 3585,\n 6231,\n 1997,\n 1996,\n 6261,\n 2984,\n 1012,\n 3202,\n 1999,\n 2392,\n 1997,\n 1996,\n 2364,\n 2311,\n 1998,\n 5307,\n 2009,\n 1010,\n 2003,\n 1037,\n 6967,\n 6231,\n 1997,\n 4828,\n 2007,\n 2608,\n 2039,\n 14995,\n 6924,\n 2007,\n 1996,\n 5722,\n 1000,\n 2310,\n 3490,\n 2618,\n 4748,\n 2033,\n 18168,\n 5267,\n 1000,\n 1012,\n 2279,\n 2000,\n 1996,\n 2364,\n 2311,\n 2003,\n 1996,\n 13546,\n 1997,\n 1996,\n 6730,\n 2540,\n 1012,\n 3202,\n 2369,\n 1996,\n 13546,\n 2003,\n 1996,\n 24665,\n 23052,\n 1010,\n 1037,\n 14042,\n 2173,\n 1997,\n 7083,\n 1998,\n 9185,\n 1012,\n 2009,\n 2003,\n 1037,\n 15059,\n 1997,\n 1996,\n 24665,\n 23052,\n 2012,\n 10223,\n 26371,\n 1010,\n 2605,\n 2073,\n 1996,\n 6261,\n 2984,\n 22353,\n 2135,\n 2596,\n 2000,\n 3002,\n 16595,\n 9648,\n 4674,\n 2061,\n 12083,\n 9711,\n 2271,\n 1999,\n 8517,\n 1012,\n 2012,\n 1996,\n 2203,\n 1997,\n 1996,\n 2364,\n 3298,\n 1006,\n 1998,\n 1999,\n 1037,\n 3622,\n 2240,\n 2008,\n 8539,\n 2083,\n 1017,\n 11342,\n 1998,\n 1996,\n 2751,\n 8514,\n 1007,\n 1010,\n 2003,\n 1037,\n 3722,\n 1010,\n 2715,\n 2962,\n 6231,\n 1997,\n 2984,\n 1012,\n 102]"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example = datasets['train'][0]\n",
    "tokenized_example = tokenizer(example['question'],example['context'])\n",
    "tokenized_example['input_ids']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "[None,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n None,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n 1,\n None]"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence_ids = tokenized_example.sequence_ids()\n",
    "sequence_ids"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "max_length = 384\n",
    "doc_stride = 128\n",
    "\n",
    "tokenized_example = tokenizer(\n",
    "    example['question'],\n",
    "    example['context'],\n",
    "    max_length = max_length,\n",
    "    truncation = 'only_second',\n",
    "    return_overflowing_tokens = True,\n",
    "    stride = doc_stride\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "[176]"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(x) for x in tokenized_example['input_ids']]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "切片： 0\n",
      "[CLS] to whom did the virgin mary allegedly appear in 1858 in lourdes france? [SEP] architecturally, the school has a catholic character. atop the main building's gold dome is a golden statue of the virgin mary. immediately in front of the main building and facing it, is a copper statue of christ with arms upraised with the legend \" venite ad me omnes \". next to the main building is the basilica of the sacred heart. immediately behind the basilica is the grotto, a marian place of prayer and reflection. it is a replica of the grotto at lourdes, france where the virgin mary reputedly appeared to saint bernadette soubirous in 1858. at the end of the main drive ( and in a direct line that connects through 3 statues and the gold dome ), is a simple, modern stone statue of mary. [SEP]\n"
     ]
    }
   ],
   "source": [
    "for i,x in enumerate(tokenized_example[\"input_ids\"][:2]):\n",
    "    print(\"切片： {}\".format(i))\n",
    "    print(tokenizer.decode(x))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "tokenizer_example = tokenizer(\n",
    "    example['question'],\n",
    "    example['context'],\n",
    "    max_length = max_length,\n",
    "    truncation='only_second',\n",
    "    return_overflowing_tokens = True,\n",
    "    return_offsets_mapping = True,\n",
    "    stride = doc_stride\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0), (0, 2), (3, 7), (8, 11), (12, 15), (16, 22), (23, 27), (28, 37), (38, 44), (45, 47), (48, 52), (53, 55), (56, 59), (59, 63), (64, 70), (70, 71), (0, 0), (0, 13), (13, 15), (15, 16), (17, 20), (21, 27), (28, 31), (32, 33), (34, 42), (43, 52), (52, 53), (54, 58), (59, 62), (63, 67)]\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer_example['offset_mapping'][0][:30])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "answers = example['answers']\n",
    "start_char = answers[\"answer_start\"][0]\n",
    "end_char = start_char + len(answers['text'][0])\n",
    "\n",
    "tokenn_start_index = 0\n",
    "while sequence_ids[tokenn_start_index] != 1:\n",
    "    tokenn_start_index -= 1\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}